﻿/*************************************************************
 * Software of blackcore
 *
 * CLRVersion:4.0.30319.586
 * MachineName:SPACE
 * Author:Andy Huang
 * Email:itblackhole@gmail.com
 * DateTime：6/8/2013 2:31:52 PM
 * Function：
 *************************************************************/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using NLog;
using System.Collections;
using System.ComponentModel;
using System.Diagnostics;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading;
using System.IO.Compression;

namespace Sob.Common
{
    /// <summary>
    /// 抓取网页
    /// </summary>
    public class CrawlPage
    {
        #region ******************************Private Property******************************
        private static Logger logger = LogManager.GetCurrentClassLogger();

        private CookieContainer cookieContainer;
        private CookieCollection cookieCollection;

        private Uri uri;
        /// <summary>
        /// 是否执行抓取操作
        /// </summary>
        private bool isCrawl = true;
        #endregion ******************************Private Property******************************


        #region ******************************Public Property******************************
        /// <summary>
        /// 页面抓取间隔
        /// <para>ms为单位，默认为0</para>
        /// </summary>
        public int CrawlInterval { get; set; }

        /// <summary>
        /// 抓取网页结果信息
        /// </summary>
        public PageInfo PageInfo { get; private set; }
        #endregion ******************************Public Property******************************


        #region ******************************Structure******************************
        public CrawlPage()
        {
            cookieContainer = new CookieContainer();
            cookieCollection = new CookieCollection();

            CrawlInterval = 0;
            PageInfo = new PageInfo();
        }

        public CrawlPage(string url)
            : this()
        {
            ResetRequestUrl(url);
        }

        public CrawlPage(Uri uri)
            : this()
        {
            this.uri = uri;
            this.PageInfo.RequestUri = this.uri;
        }
        #endregion  ******************************Structure******************************


        #region ******************************Private Method******************************

        /// <summary>
        /// 创建UserAgent列表
        /// <para>可参照：http://www.useragentstring.com/pages/All/</para>
        /// </summary>        
        /// <returns></returns>
        [Description("随机生成的UserAgent直接导致同一个页面返回的页面内容格式也会不同")]
        private string CreateUserAgent()
        {
            int i = 1;
            Hashtable userAgents = new Hashtable();

            //本机FireFox：测试通过Google,Baidu
            userAgents.Add(i++, "Mozilla/5.0 (Windows NT 6.1; rv:8.0.1) Gecko/20100101 Firefox/8.0.1");
            userAgents.Add(i++, string.Format("Mozilla/5.0 (Windows NT 6.1; rv:8.0.1) Gecko/20100101{0} Firefox/8.0.1", Guid.NewGuid().ToString("N")));//随机变化UserAgent
            userAgents.Add(i++, "Mozilla/5.0 (Windows NT 6.0; rv:12.0) Gecko/20100101 Firefox/12.0");

            //本机Internet Explorer：测试通过Google,Baidu
            userAgents.Add(i++, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; BRI/2; IEShow Toolbar; IEShow wangsuToolBar; Tablet PC 2.0; InfoPath.3; chromeframe/17.0.963.56)");
            userAgents.Add(i++, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Trident/5.0; SLCC1; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729)");

            //本机BaiduBrowser：测试通过Google,Baidu
            userAgents.Add(i++, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; BRI/2; IEShow Toolbar; IEShow wangsuToolBar; Tablet PC 2.0; InfoPath.3; baidubrowser 1.x)");

            //其它
            userAgents.Add(i++, "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36");

            Random rnd = new Random();
            int key = rnd.Next(1, userAgents.Count + 1);
            return userAgents[key].ToString();
        }

        private bool UrlToUri(string url)
        {
            if (!string.IsNullOrWhiteSpace(url))
            {
                try
                {
                    this.uri = new Uri(url);
                    this.PageInfo.RequestUri = this.uri;
                    return true;
                }
                catch (Exception err)
                {
                    isCrawl = false;
                    PageInfo.IsSuccess = false;
                    PageInfo.Exception = err;
                }
            }
            return false;
        }

        private void ResetRequestUrl(string url = "")
        {
            if (string.IsNullOrWhiteSpace(url) && this.uri == null)
            {
                isCrawl = false;
                this.PageInfo.IsSuccess = false;
                this.PageInfo.Exception = new Exception("访问URL不能为空！");
            }
            else if ((this.uri == null && !string.IsNullOrWhiteSpace(url)) || (this.uri != null && !string.IsNullOrWhiteSpace(url)))
            {
                //优先使用DoanloadPage中的参数
                UrlToUri(url);
            }
        }

        /// <summary>
        /// 获取网页详细信息
        /// <para>HttpWebResponse.ContentEncoding是标识这个内容是什么编码的，而HttpWebResponse.Charset 是告诉客户端怎么显示的，不设置Charset时会使用ContentEncoding的值</para>
        /// </summary>      
        /// <param name="pageByteContent"></param>
        /// <returns></returns>
        private void GetPageContent(HttpWebResponse response, byte[] pageByteContent)
        {
            //获取页面编码
            Encoding currentEncoding = Encoding.GetEncoding(response.CharacterSet);
            string contentType = response.ContentType.ToLower();
            int isExistCharset = contentType.IndexOf("charset=");
            if (isExistCharset > -1)
            {
                currentEncoding = Encoding.GetEncoding(response.ContentType.Substring(isExistCharset + "charset".Length + 1));
            }
            this.PageInfo.Charset = currentEncoding;
            this.PageInfo.Snapshot = currentEncoding.GetString(pageByteContent);

            //修正内容编码：编码为ISO-8859-1时，则有可能为GB2312或UTF-8编码            
            if (currentEncoding.BodyName.Trim().ToUpper().Equals("ISO-8859-1"))
            {
                this.PageInfo.Charset = GetEncodingByISO88591(pageByteContent);
                //用新的编码解析内容
                this.PageInfo.Snapshot = this.PageInfo.Charset.GetString(pageByteContent);
            }

            //this.PageInfo.Content = new HtmlHelper().RemoveHTMLTags(this.PageInfo.Snapshot);
            this.PageInfo.ContentLength = this.PageInfo.Snapshot.Length;
            this.PageInfo.Title = GetPageTitle(this.PageInfo.Snapshot);
            this.PageInfo.Keywords = GetPageKeyword(this.PageInfo.Snapshot);
            this.PageInfo.Description = GetPageDescription(this.PageInfo.Snapshot);
        }

        /// <summary>
        /// 将ISO88591转码为UTF8或GB2312
        /// <para>Response中的编码是ISO-8859-1时，在转换后的内容页中取charset的值再进行编码</para>
        /// </summary>        
        /// <returns></returns>
        private Encoding GetEncodingByISO88591(byte[] pageByteContent)
        {
            Encoding pageContentEncoding = Encoding.GetEncoding("ISO-8859-1");
            string iso88591Content = pageContentEncoding.GetString(pageByteContent);
            //Regex regex = new Regex("charset=(?<charset>[\\s\\S]*?)[\"|']");
            //Match match = regex.Match(iso88591Content);
            //if (match.Success)
            //{
            //    pageContentEncoding = Encoding.GetEncoding(match.Groups["charset"].Value);
            //}

            string charsetFlag = "charset=";
            string pattern = charsetFlag + "(?<charset>[\\s\\S]*?)[\"|'|/>]";
            int startIndex = iso88591Content.IndexOf(charsetFlag, 0, iso88591Content.Length > 512 ? 512 : iso88591Content.Length);
            if (startIndex >= 0)
            {
                startIndex += charsetFlag.Length;
                string flagCharacter = iso88591Content.Substring(startIndex, 1);
                if (flagCharacter.Equals("'"))
                {
                    pattern = charsetFlag + "'(?<charset>[\\s\\S]*?)[\"|']";
                }
                else if (flagCharacter.Equals("\""))
                {
                    pattern = charsetFlag + "\"(?<charset>[\\s\\S]*?)[\"|']";
                }
                Regex r = new Regex(pattern);
                Match m = r.Match(iso88591Content);
                if (m.Success)
                {
                    pageContentEncoding = Encoding.GetEncoding(m.Groups["charset"].Value.Trim());
                }
            }

            return pageContentEncoding;
        }
        #endregion ******************************Private Method******************************


        #region ******************************Public Method******************************
        /// <summary>
        /// 下载指定网页内容
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public bool DownloadPage(string url = "")
        {
            ResetRequestUrl(url);
            if (!isCrawl)
            {
                return false;
            }

            Stopwatch sw = new Stopwatch();
            sw.Start();

            try
            {
                Thread.Sleep(CrawlInterval);
                //----------------------------------读数据---------------------------
                HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(this.uri.AbsoluteUri);
                req.Timeout = 30000;
                req.Method = "GET";
                string userAgent = CreateUserAgent();
                req.UserAgent = userAgent;
                req.KeepAlive = false;//默认为true，设置为false,用完就及时关闭               
                req.MaximumAutomaticRedirections = 9999;
                req.AllowAutoRedirect = true;
                req.Headers.Add("Accept-Encoding", "gzip,deflate,sdch");//让服务器返回数据时进行gzip压缩，减少数据包大小以提高数据传输速率而提高性能

                //测试参数
                //req.CookieContainer = ccon;//默认就是禁用cookie,不要设置cookie                                
                //req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";                                         
                //测试参数


                using (HttpWebResponse response = req.GetResponse() as HttpWebResponse)
                {
                    this.PageInfo.ContentType = response.ContentType;
                    this.PageInfo.ResponseUri = response.ResponseUri;
                    //获取头信息集合
                    //WebHeaderCollection headers = response.Headers;
                    //如果在header设置了Transfer-Encoding：chunked则response.ContentLength会返回-1
                    //if (!(response.ContentType.ToString().StartsWith("text/") && response.ContentLength > 0))
                    if (!response.ContentType.ToString().StartsWith("text"))
                    {
                        if (response.ContentType.ToString().StartsWith("application/octet-stream"))
                        {

                        }
                        else if (response.ContentType.ToString().StartsWith("image/jpeg"))
                        {

                        }
                        this.PageInfo.Exception = new Exception(string.Format("response.ContentType：{0}.\t{1}", response.ContentType, response.ResponseUri.AbsoluteUri));
                        return false;
                    }

                    byte[] pageByteContent = null;
                    //获取数据
                    if (response.ContentEncoding.Equals("gzip"))
                    {
                        //需要解压
                        using (Stream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            pageByteContent = GetResponseStream(stream);
                        }
                    }
                    else
                    {
                        //无须解压
                        using (Stream stream = response.GetResponseStream())
                        {
                            pageByteContent = GetResponseStream(stream);
                        }
                    }

                    if (pageByteContent == null)
                    {
                        this.PageInfo.IsSuccess = false;
                        this.PageInfo.Exception = new Exception("读取内容为空！");
                        return false;
                    }
                    else
                    {
                        GetPageContent(response, pageByteContent);
                        this.PageInfo.IsSuccess = true;
                        return true;
                    }
                }
            }
            catch (WebException err)
            {
                logger.Info("------>WebException:{0}/1", sw.Elapsed.ToString(), err.ToString());
                logger.Error("----------Url：'{0}'，\r\n----------Excpetion：{1}.", uri.AbsoluteUri, err.ToString());
                PageInfo.Exception = err;
            }
            catch (Exception err)
            {
                logger.Info("------>Exception:{0}", sw.Elapsed.ToString());
                logger.Error("----------Url：'{0}'，\r\n----------Excpetion：{1}.", uri.AbsoluteUri, err.ToString());
                this.PageInfo.Exception = err;
            }
            finally
            {
                this.PageInfo.UseTime = sw;
                sw.Stop();
                logger.Info("------>Finished:{0}", sw.Elapsed.ToString());
            }
            return false;
        }

        /// <summary>
        /// 获取Response数据流
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        private byte[] GetResponseStream(Stream stream)
        {
            byte[] pageByteContent = null;
            int totalByteLength = 0;
            byte[] bytes = new byte[1024];

            using (MemoryStream memoryStream = new MemoryStream())
            {
                int currentLength = 0;
                currentLength = stream.Read(bytes, 0, 1024);
                totalByteLength += currentLength;

                while (currentLength > 0)
                {
                    memoryStream.Write(bytes, 0, currentLength);
                    currentLength = stream.Read(bytes, 0, 1024);
                    totalByteLength += currentLength;
                }
                pageByteContent = memoryStream.ToArray();
            }

            return pageByteContent;
        }



        /// <summary>
        /// 获取页面内容的标题
        /// <para>&lt;title&gt;标题&lt;/title&gt;</para>
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string GetPageTitle(string content)
        {
            string title = string.Empty;
            Regex regex = new Regex(@"(?m)<title[^>]*>(?<title>(?:\w|\W)*?)</title[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase);
            Match match = regex.Match(content);
            if (match.Success)
            {
                title = match.Groups["title"].Value;
            }
            if (string.IsNullOrWhiteSpace(title))
            {
                title = "TitleIsEmpty." + DateTime.Now.Ticks;
            }
            return title;
        }

        /// <summary>
        /// 获取页面内容的关键词列表
        /// <para>&lt;meta name="keywords" content="关键词列表"</para>
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string GetPageKeyword(string content)
        {
            string keyword = string.Empty;
            Regex regex = new Regex(@"<meta\s+name=""keywords""\s+content=\""(?<keywords>[\s\S]*?)\""", RegexOptions.Multiline | RegexOptions.IgnoreCase);
            Match match = regex.Match(content);
            if (match.Success)
            {
                keyword = match.Groups["keywords"].Value;
            }
            if (string.IsNullOrWhiteSpace(keyword))
            {
                keyword = "KeywordsIsEmpty." + DateTime.Now.Ticks;
            }
            return keyword;
        }

        /// <summary>
        /// 获取页面内容的描述信息
        /// <para>&lt;meta name="description" content="描述信息"</para>
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string GetPageDescription(string content)
        {
            string description = string.Empty;
            Regex regex = new Regex(@"<meta\s+name=""description""\s+content=\""(?<description>[\s\S]*?)\""", RegexOptions.Multiline | RegexOptions.IgnoreCase);
            Match match = regex.Match(content);
            if (match.Success)
            {
                description = match.Groups["description"].Value;
            }
            if (string.IsNullOrWhiteSpace(description))
            {
                description = "DescriptionIsEmpty." + DateTime.Now.Ticks;
            }
            return description;
        }


        #endregion ******************************Public Method******************************
    }
}
